#!/usr/bin/python
# -*- coding: utf-8 -*-
# From: ubuntu.org.cn Copyright: GPLv2
import thread        
import urllib2
import sys, re
import os
from datetime import datetime

class findproxy(object):
  """find usable proxies from "proxyurl", using "pattern".
  connect to "desturl" and check if there is a word "checkvalue" in it.
  """
  def __init__(self):
    object.__init__(self)
#    self.data = {"proxyurl":"http://www.youdaili.net/Daili/http/2814.html",
 #     "pattern":"(\d{2,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d{2,5})",

    self.data = {"proxyurl":"http://www.kuaidaili.com",
      "pattern":"<td data-title=\"IP\">(\d{2,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"
                "</td>\s+<td data-title=\"PORT\">(\d{2,5})</td>",

#"desturl":"http://www.sogou.com/",
      "desturl":"http://weixin.sogou.com/weixin?query=%E7%9A%84&_asf=www.sogou.com&_ast=1418190471&w=01019900&p=40040100&ie=utf8&type=2",
#"checkvalue":"搜狗搜索引擎 - 上网从搜狗开始",
      "checkvalue":"微信公众平台",
      "output":"proxy.txt"}
    self.ipdict = {}
    self.passeddict = {}
    print "init success"

  def getlist(self):
    "get a proxy list from proxyurl."
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0'}
    req = urllib2.Request(url=self.data["proxyurl"], headers=headers)
    sock = urllib2.urlopen(req)
    html = sock.read()
    print html
    sock.close()
    for ip in re.compile(self.data["pattern"]).findall(html):
      self.ipdict[":".join(ip)] = -1
      print ip

  def hander(self, ip):
    "check if the ip address is usable(one of the threads)."
    try:
      start = datetime.now()
      proxy = urllib2.ProxyHandler({'http':ip})
      opener = urllib2.build_opener(proxy)
      urllib2.install_opener(opener)
      sock = urllib2.urlopen(self.data["desturl"])
      end = datetime.now()            
      html = sock.read()
      sock.close()
        self.ipdict[ip] = self.passeddict[ip] = end - start
      if self.data["checkvalue"] in html:
      else:
        self.ipdict[ip] = "不符合条件"
    except:
      self.ipdict[ip] = "连接失败"
    finally:
      print "%-25s%s" % (ip, self.ipdict[ip])

  def check(self):
    "create threads to check"
    for ip in self.ipdict.keys():
      thread.start_new_thread(self.hander, (ip,))

  def savelist(self):
    'save usable proxies to "filename"'
    sortedlist = sorted(self.passeddict.items(), key = lambda x:x[1])
    i = 0
    outputlist = []
    for (ip, t) in sortedlist:
      if i < 20:
        outputlist.append(ip)
        i = i + 1
    if i == 0:
      return

    try:
      logfile = open(self.data["output"], "w")
      #logfile.write("".join(["%-25s\n" % (ip) for (ip, t) in sortedlist]))
      logfile.write("".join(["%-25s\n" % (ip) for (ip) in outputlist]))
      #os.system('cp /var/yr/static/yrdata/bi/crawl/web/fetcher/proxy.txt.bak /var/yr/static/yrdata/bi/crawl/web/fetcher/proxy.txt')

    finally:
      logfile.close()

def main(args = sys.argv[1:]):
  import getopt
  import socket
  helpstr = """
  FindProxy 0.0.2
  Usage:
  -p(--proxy)     the url to get proxies from.
  -r(--re)        the regexp which the IP string must meet.
  -d(--dest)      the destination url to reach through proxies.
  -v(--value)     the value to check if the proxies return correct data.
  -h(--help)      show this message.
  """
  Proxy = findproxy()
  socket.setdefaulttimeout(5)
  print "get list"
  Proxy.getlist()
  Proxy.check()
  '''
  try:
    opts = getopt.getopt(args, "p:r:d:v:o:h", ["proxy=", "re=", "dest=", "value=", "output=", "help"])[0]
  except getopt.GetoptError:
    sys.exit(helpstr)
  for o, a in opts:
    if o in ("-h", "--help"):
      sys.exit(helpstr)
    d = {("-p", "--proxy"):"proxyurl",
      ("-r", "--re"):"pattern",
      ("-d", "--dest"):"desturl",
      ("-v", "--value"):"checkvalue",
      ("-o", "--output"):"output"}
    for opt, attr in d.items():
      if o in opt:
        Proxy.data[attr] = a
        socket.setdefaulttimeout(5)
        Proxy.getlist()
        Proxy.check()
    '''
  #Wait for all threads quited
  while -1 in Proxy.ipdict.values():
    pass
  print 'Succeed!!! All available proxies will be saved in "%s".' % Proxy.data["output"]
  Proxy.savelist()    

if __name__ == "__main__":
  main()
